{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Multi-Horizon Forecasting for Limit Order Books: Novel Deep Learning Approaches and Hardware Acceleration using Intelligent Processing Units\n",
    "### Authors: Zihao Zhang and Stefan Zohren\n",
    "### Oxford-Man Institute of Quantitative Finance, Department of Engineering Science, University of Oxford\n",
    "\n",
    "This jupyter notebook is used to demonstrate our recent paper [2]. We use FI-2010 [1] dataset and present how model architecture is constructed here. The FI-2010 is publicly avilable and interested readers can check out their paper [1]. The dataset can be downloaded from: https://etsin.fairdata.fi/dataset/73eb48d7-4dbc-4a10-a52a-da745b47a649\n",
    "\n",
    "Otherwise, it can be obtained from: https://drive.google.com/drive/folders/1Xen3aRid9ZZhFqJRgEMyETNazk02cNmv?usp=sharing\n",
    "\n",
    "[1] Ntakaris A, Magris M, Kanniainen J, Gabbouj M, Iosifidis A. Benchmark dataset for mid‐price forecasting of limit order book data with machine learning methods. Journal of Forecasting. 2018 Dec;37(8):852-66. https://arxiv.org/abs/1705.03233\n",
    "\n",
    "[2] Zhang Z, Zohren S. Multi-Horizon Forecasting for Limit Order Books: Novel Deep Learning Approaches and Hardware Acceleration using Intelligent Processing Units. https://arxiv.org/abs/2105.10430\n",
    "\n",
    "#### This notebook demonstrates how to train DeepLOB-Seq2Seq by using tensorflow 2 on IPUs.\n",
    "\n",
    "#### For more information about IPU, please check https://www.graphcore.ai/\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import absolute_import, division, print_function, unicode_literals\n",
    "\n",
    "import os\n",
    "import logging\n",
    "import glob\n",
    "import argparse\n",
    "import sys\n",
    "import time\n",
    "import tensorflow as tf\n",
    "from tensorflow.python import ipu\n",
    "from sklearn.metrics import accuracy_score, classification_report\n",
    "import pandas as pd\n",
    "import pickle\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from collections import Counter\n",
    "\n",
    "# load my packages\n",
    "from preprocess import *\n",
    "from model import get_model_seq, get_model_attention"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# please change the data_path to your local path\n",
    "data_path = '/home/zihaoz/deeplob/data'\n",
    "\n",
    "T = 50 # lookback window size\n",
    "epochs = 150 # number of training epochs\n",
    "batch_size = 16 # gradient descent batch size\n",
    "n_hidden = 64 # hidden state for decoder\n",
    "SHUFFLE=True # shuffle the traning data\n",
    "saved_model_path = './model_deeplob_seq/deeplob_seq' # saved model path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_encoder_input.shape = (254701, 50, 40, 1),train_decoder_target.shape = (254701, 5, 3)\n",
      "test_encoder_input.shape = (139538, 50, 40, 1),test_decoder_target.shape = (139538, 5, 3)\n"
     ]
    }
   ],
   "source": [
    "# load data\n",
    "dec_train = np.loadtxt(data_path + '/Train_Dst_NoAuction_DecPre_CF_7.txt')\n",
    "dec_test1 = np.loadtxt(data_path + '/Test_Dst_NoAuction_DecPre_CF_7.txt')\n",
    "dec_test2 = np.loadtxt(data_path + '/Test_Dst_NoAuction_DecPre_CF_8.txt')\n",
    "dec_test3 = np.loadtxt(data_path + '/Test_Dst_NoAuction_DecPre_CF_9.txt')\n",
    "dec_test = np.hstack((dec_test1, dec_test2, dec_test3))\n",
    "\n",
    "# extract limit order book data from the FI-2010 dataset\n",
    "train_lob = prepare_x(dec_train)\n",
    "test_lob = prepare_x(dec_test)\n",
    "\n",
    "# extract label from the FI-2010 dataset\n",
    "train_label = get_label(dec_train)\n",
    "test_label = get_label(dec_test)\n",
    "\n",
    "# prepare training data. We feed past T observations into our algorithms.\n",
    "train_encoder_input, train_decoder_target = data_classification(train_lob, train_label, T)\n",
    "train_decoder_input = prepare_decoder_input(train_encoder_input, teacher_forcing=False)\n",
    "\n",
    "test_encoder_input, test_decoder_target = data_classification(test_lob, test_label, T)\n",
    "test_decoder_input = prepare_decoder_input(test_encoder_input, teacher_forcing=False)\n",
    "\n",
    "print(f'train_encoder_input.shape = {train_encoder_input.shape},'\n",
    "      f'train_decoder_target.shape = {train_decoder_target.shape}')\n",
    "print(f'test_encoder_input.shape = {test_encoder_input.shape},'\n",
    "      f'test_decoder_target.shape = {test_decoder_target.shape}')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Configure the IPU system\n",
    "cfg = ipu.utils.create_ipu_config()\n",
    "cfg = ipu.utils.auto_select_ipus(cfg, 1)\n",
    "ipu.utils.configure_ipu_system(cfg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "15918/15918 [==============================] - 140s 9ms/step - loss: 1.0462 - accuracy: 0.4472\n",
      "Epoch 2/5\n",
      "15918/15918 [==============================] - 52s 3ms/step - loss: 1.0210 - accuracy: 0.4659\n",
      "Epoch 3/5\n",
      "15918/15918 [==============================] - 52s 3ms/step - loss: 0.9553 - accuracy: 0.5052\n",
      "Epoch 4/5\n",
      "15918/15918 [==============================] - 59s 4ms/step - loss: 0.9297 - accuracy: 0.5207\n",
      "Epoch 5/5\n",
      "15918/15918 [==============================] - 51s 3ms/step - loss: 0.9179 - accuracy: 0.5278\n",
      "Epoch = 5,Validation Results = [1.0359435021652577, 0.47807884]\n",
      "Epoch 6/10\n",
      "15918/15918 [==============================] - 138s 9ms/step - loss: 0.9096 - accuracy: 0.5325\n",
      "Epoch 7/10\n",
      "15918/15918 [==============================] - 52s 3ms/step - loss: 0.9030 - accuracy: 0.5370\n",
      "Epoch 8/10\n",
      "15918/15918 [==============================] - 52s 3ms/step - loss: 0.8974 - accuracy: 0.5396\n",
      "Epoch 9/10\n",
      "15918/15918 [==============================] - 59s 4ms/step - loss: 0.8927 - accuracy: 0.5421\n",
      "Epoch 10/10\n",
      "15918/15918 [==============================] - 51s 3ms/step - loss: 0.8870 - accuracy: 0.5447\n",
      "Epoch = 10,Validation Results = [1.0029647791260188, 0.48766887]\n",
      "Epoch 11/15\n",
      "15918/15918 [==============================] - 136s 9ms/step - loss: 0.8831 - accuracy: 0.5474\n",
      "Epoch 12/15\n",
      "15918/15918 [==============================] - 52s 3ms/step - loss: 0.8792 - accuracy: 0.5499\n",
      "Epoch 13/15\n",
      "15918/15918 [==============================] - 52s 3ms/step - loss: 0.8748 - accuracy: 0.5514\n",
      "Epoch 14/15\n",
      "15918/15918 [==============================] - 58s 4ms/step - loss: 0.8720 - accuracy: 0.5530\n",
      "Epoch 15/15\n",
      "15918/15918 [==============================] - 51s 3ms/step - loss: 0.8692 - accuracy: 0.5553\n",
      "Epoch = 15,Validation Results = [0.9052797757730501, 0.5411679]\n",
      "Epoch 16/20\n",
      "15918/15918 [==============================] - 141s 9ms/step - loss: 0.8660 - accuracy: 0.5564\n",
      "Epoch 17/20\n",
      "15918/15918 [==============================] - 52s 3ms/step - loss: 0.8635 - accuracy: 0.5579\n",
      "Epoch 18/20\n",
      "15918/15918 [==============================] - 52s 3ms/step - loss: 0.8603 - accuracy: 0.5595\n",
      "Epoch 19/20\n",
      "15918/15918 [==============================] - 60s 4ms/step - loss: 0.8599 - accuracy: 0.5602\n",
      "Epoch 20/20\n",
      "15918/15918 [==============================] - 51s 3ms/step - loss: 0.8566 - accuracy: 0.5625\n",
      "Epoch = 20,Validation Results = [0.9296963038780275, 0.5376767]\n",
      "Epoch 21/25\n",
      "15918/15918 [==============================] - 140s 9ms/step - loss: 0.8544 - accuracy: 0.5633\n",
      "Epoch 22/25\n",
      "15918/15918 [==============================] - 52s 3ms/step - loss: 0.8528 - accuracy: 0.5648\n",
      "Epoch 23/25\n",
      "15918/15918 [==============================] - 52s 3ms/step - loss: 0.8509 - accuracy: 0.5650\n",
      "Epoch 24/25\n",
      "15918/15918 [==============================] - 58s 4ms/step - loss: 0.8485 - accuracy: 0.5675\n",
      "Epoch 25/25\n",
      "15918/15918 [==============================] - 51s 3ms/step - loss: 0.8470 - accuracy: 0.5681\n",
      "Epoch = 25,Validation Results = [0.8765886861279705, 0.55609095]\n",
      "Epoch 26/30\n",
      "15918/15918 [==============================] - 137s 9ms/step - loss: 0.8461 - accuracy: 0.5694\n",
      "Epoch 27/30\n",
      "15918/15918 [==============================] - 51s 3ms/step - loss: 0.8427 - accuracy: 0.5715\n",
      "Epoch 28/30\n",
      "15918/15918 [==============================] - 51s 3ms/step - loss: 0.8429 - accuracy: 0.5721\n",
      "Epoch 29/30\n",
      "15918/15918 [==============================] - 57s 4ms/step - loss: 0.8411 - accuracy: 0.5740\n",
      "Epoch 30/30\n",
      "15918/15918 [==============================] - 52s 3ms/step - loss: 0.8390 - accuracy: 0.5758\n",
      "Epoch = 30,Validation Results = [0.9112871621392767, 0.54719603]\n",
      "Epoch 31/35\n",
      "15918/15918 [==============================] - 137s 9ms/step - loss: 0.8381 - accuracy: 0.5782\n",
      "Epoch 32/35\n",
      "15918/15918 [==============================] - 52s 3ms/step - loss: 0.8353 - accuracy: 0.5831\n",
      "Epoch 33/35\n",
      "15918/15918 [==============================] - 51s 3ms/step - loss: 0.8332 - accuracy: 0.5878\n",
      "Epoch 34/35\n",
      "15918/15918 [==============================] - 59s 4ms/step - loss: 0.8303 - accuracy: 0.5935\n",
      "Epoch 35/35\n",
      "15918/15918 [==============================] - 51s 3ms/step - loss: 0.8267 - accuracy: 0.6010\n",
      "Epoch = 35,Validation Results = [0.8559668276454883, 0.5727773]\n",
      "Epoch 36/40\n",
      "15918/15918 [==============================] - 135s 8ms/step - loss: 0.8216 - accuracy: 0.6091\n",
      "Epoch 37/40\n",
      "15918/15918 [==============================] - 52s 3ms/step - loss: 0.8142 - accuracy: 0.6187\n",
      "Epoch 38/40\n",
      "15918/15918 [==============================] - 51s 3ms/step - loss: 0.8064 - accuracy: 0.6284\n",
      "Epoch 39/40\n",
      "15918/15918 [==============================] - 59s 4ms/step - loss: 0.7968 - accuracy: 0.6383\n",
      "Epoch 40/40\n",
      "15918/15918 [==============================] - 52s 3ms/step - loss: 0.7875 - accuracy: 0.6464\n",
      "Epoch = 40,Validation Results = [0.8471099839110116, 0.5897149]\n",
      "Epoch 41/45\n",
      "15918/15918 [==============================] - 143s 9ms/step - loss: 0.7803 - accuracy: 0.6521\n",
      "Epoch 42/45\n",
      "15918/15918 [==============================] - 51s 3ms/step - loss: 0.7734 - accuracy: 0.6572\n",
      "Epoch 43/45\n",
      "15918/15918 [==============================] - 50s 3ms/step - loss: 0.7667 - accuracy: 0.6614\n",
      "Epoch 44/45\n",
      "15918/15918 [==============================] - 58s 4ms/step - loss: 0.7610 - accuracy: 0.6659\n",
      "Epoch 45/45\n",
      "15918/15918 [==============================] - 51s 3ms/step - loss: 0.7541 - accuracy: 0.6701\n",
      "Epoch = 45,Validation Results = [0.8557877750845495, 0.5986373]\n",
      "Epoch 46/50\n",
      "15918/15918 [==============================] - 138s 9ms/step - loss: 0.7495 - accuracy: 0.6734\n",
      "Epoch 47/50\n",
      "15918/15918 [==============================] - 51s 3ms/step - loss: 0.7455 - accuracy: 0.6762\n",
      "Epoch 48/50\n",
      "15918/15918 [==============================] - 51s 3ms/step - loss: 0.7386 - accuracy: 0.6802\n",
      "Epoch 49/50\n",
      "15918/15918 [==============================] - 59s 4ms/step - loss: 0.7341 - accuracy: 0.6834\n",
      "Epoch 50/50\n",
      "15918/15918 [==============================] - 51s 3ms/step - loss: 0.7314 - accuracy: 0.6852\n",
      "Epoch = 50,Validation Results = [0.8651632488504206, 0.6008404]\n",
      "Epoch 51/55\n",
      "15918/15918 [==============================] - 142s 9ms/step - loss: 0.7266 - accuracy: 0.6879\n",
      "Epoch 52/55\n",
      "15918/15918 [==============================] - 52s 3ms/step - loss: 0.7225 - accuracy: 0.6903\n",
      "Epoch 53/55\n",
      "15918/15918 [==============================] - 51s 3ms/step - loss: 0.7194 - accuracy: 0.6924\n",
      "Epoch 54/55\n",
      "15918/15918 [==============================] - 59s 4ms/step - loss: 0.7157 - accuracy: 0.6946\n",
      "Epoch 55/55\n",
      "15918/15918 [==============================] - 50s 3ms/step - loss: 0.7116 - accuracy: 0.6974\n",
      "Epoch = 55,Validation Results = [0.8191566359090227, 0.6208373]\n",
      "Epoch 56/60\n",
      "15918/15918 [==============================] - 137s 9ms/step - loss: 0.7096 - accuracy: 0.6982\n",
      "Epoch 57/60\n",
      "15918/15918 [==============================] - 52s 3ms/step - loss: 0.7058 - accuracy: 0.7006\n",
      "Epoch 58/60\n",
      "15918/15918 [==============================] - 52s 3ms/step - loss: 0.7015 - accuracy: 0.7031\n",
      "Epoch 59/60\n",
      "15918/15918 [==============================] - 57s 4ms/step - loss: 0.6998 - accuracy: 0.7035\n",
      "Epoch 60/60\n",
      "15918/15918 [==============================] - 51s 3ms/step - loss: 0.6970 - accuracy: 0.7056\n",
      "Epoch = 60,Validation Results = [0.8560758092207469, 0.6068371]\n",
      "Epoch 61/65\n",
      "15918/15918 [==============================] - 141s 9ms/step - loss: 0.6937 - accuracy: 0.7073\n",
      "Epoch 62/65\n",
      "15918/15918 [==============================] - 52s 3ms/step - loss: 0.6901 - accuracy: 0.7091\n",
      "Epoch 63/65\n",
      "15918/15918 [==============================] - 50s 3ms/step - loss: 0.6878 - accuracy: 0.7108\n",
      "Epoch 64/65\n",
      "15918/15918 [==============================] - 60s 4ms/step - loss: 0.6861 - accuracy: 0.7116\n",
      "Epoch 65/65\n",
      "15918/15918 [==============================] - 49s 3ms/step - loss: 0.6825 - accuracy: 0.7138\n",
      "Epoch = 65,Validation Results = [0.8191574106149158, 0.6307414]\n",
      "Epoch 66/70\n",
      "15918/15918 [==============================] - 137s 9ms/step - loss: 0.6799 - accuracy: 0.7153\n",
      "Epoch 67/70\n",
      "15918/15918 [==============================] - 49s 3ms/step - loss: 0.6769 - accuracy: 0.7166\n",
      "Epoch 68/70\n",
      "15918/15918 [==============================] - 52s 3ms/step - loss: 0.6757 - accuracy: 0.7173\n",
      "Epoch 69/70\n",
      "15918/15918 [==============================] - 58s 4ms/step - loss: 0.6726 - accuracy: 0.7189\n",
      "Epoch 70/70\n",
      "15918/15918 [==============================] - 51s 3ms/step - loss: 0.6703 - accuracy: 0.7199\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch = 70,Validation Results = [0.845632915088671, 0.63105166]\n",
      "Epoch 71/75\n",
      "15918/15918 [==============================] - 140s 9ms/step - loss: 0.6679 - accuracy: 0.7214\n",
      "Epoch 72/75\n",
      "15918/15918 [==============================] - 53s 3ms/step - loss: 0.6644 - accuracy: 0.7230\n",
      "Epoch 73/75\n",
      "15918/15918 [==============================] - 51s 3ms/step - loss: 0.6636 - accuracy: 0.7232\n",
      "Epoch 74/75\n",
      "15918/15918 [==============================] - 57s 4ms/step - loss: 0.6607 - accuracy: 0.7247\n",
      "Epoch 75/75\n",
      "15918/15918 [==============================] - 51s 3ms/step - loss: 0.6581 - accuracy: 0.7260\n",
      "Epoch = 75,Validation Results = [0.7968672948113771, 0.6454642]\n",
      "Epoch 76/80\n",
      "15918/15918 [==============================] - 134s 8ms/step - loss: 0.6561 - accuracy: 0.7268\n",
      "Epoch 77/80\n",
      "15918/15918 [==============================] - 51s 3ms/step - loss: 0.6539 - accuracy: 0.7281\n",
      "Epoch 78/80\n",
      "15918/15918 [==============================] - 49s 3ms/step - loss: 0.6525 - accuracy: 0.7288\n",
      "Epoch 79/80\n",
      "15918/15918 [==============================] - 58s 4ms/step - loss: 0.6506 - accuracy: 0.7295\n",
      "Epoch 80/80\n",
      "15918/15918 [==============================] - 51s 3ms/step - loss: 0.6478 - accuracy: 0.7309\n",
      "Epoch = 80,Validation Results = [0.7814938865717298, 0.6561145]\n",
      "Epoch 81/85\n",
      "15918/15918 [==============================] - 139s 9ms/step - loss: 0.6468 - accuracy: 0.7315\n",
      "Epoch 82/85\n",
      "15918/15918 [==============================] - 49s 3ms/step - loss: 0.6440 - accuracy: 0.7326\n",
      "Epoch 83/85\n",
      "15918/15918 [==============================] - 51s 3ms/step - loss: 0.6436 - accuracy: 0.7326\n",
      "Epoch 84/85\n",
      "15918/15918 [==============================] - 58s 4ms/step - loss: 0.6408 - accuracy: 0.7341\n",
      "Epoch 85/85\n",
      "15918/15918 [==============================] - 51s 3ms/step - loss: 0.6390 - accuracy: 0.7349\n",
      "Epoch = 85,Validation Results = [0.7749453322488511, 0.65838045]\n",
      "Epoch 86/90\n",
      "15918/15918 [==============================] - 140s 9ms/step - loss: 0.6380 - accuracy: 0.7357\n",
      "Epoch 87/90\n",
      "15918/15918 [==============================] - 53s 3ms/step - loss: 0.6361 - accuracy: 0.7364\n",
      "Epoch 88/90\n",
      "15918/15918 [==============================] - 53s 3ms/step - loss: 0.6350 - accuracy: 0.7369\n",
      "Epoch 89/90\n",
      "15918/15918 [==============================] - 60s 4ms/step - loss: 0.6337 - accuracy: 0.7372\n",
      "Epoch 90/90\n",
      "15918/15918 [==============================] - 50s 3ms/step - loss: 0.6318 - accuracy: 0.7384\n",
      "Epoch = 90,Validation Results = [0.7742200728609427, 0.66035974]\n",
      "Epoch 91/95\n",
      "15918/15918 [==============================] - 137s 9ms/step - loss: 0.6308 - accuracy: 0.7389\n",
      "Epoch 92/95\n",
      "15918/15918 [==============================] - 50s 3ms/step - loss: 0.6285 - accuracy: 0.7398\n",
      "Epoch 93/95\n",
      "15918/15918 [==============================] - 51s 3ms/step - loss: 0.6283 - accuracy: 0.7400\n",
      "Epoch 94/95\n",
      "15918/15918 [==============================] - 59s 4ms/step - loss: 0.6261 - accuracy: 0.7411\n",
      "Epoch 95/95\n",
      "15918/15918 [==============================] - 51s 3ms/step - loss: 0.6253 - accuracy: 0.7414\n",
      "Epoch = 95,Validation Results = [0.7776556417379357, 0.6615614]\n",
      "Epoch 96/100\n",
      "15918/15918 [==============================] - 138s 9ms/step - loss: 0.6238 - accuracy: 0.7421\n",
      "Epoch 97/100\n",
      "15918/15918 [==============================] - 49s 3ms/step - loss: 0.6228 - accuracy: 0.7428\n",
      "Epoch 98/100\n",
      "15918/15918 [==============================] - 51s 3ms/step - loss: 0.6206 - accuracy: 0.7431\n",
      "Epoch 99/100\n",
      "15918/15918 [==============================] - 56s 4ms/step - loss: 0.6210 - accuracy: 0.7432\n",
      "Epoch 100/100\n",
      "15918/15918 [==============================] - 51s 3ms/step - loss: 0.6187 - accuracy: 0.7443\n",
      "Epoch = 100,Validation Results = [0.770497651477533, 0.664656]\n",
      "Epoch 101/105\n",
      "15918/15918 [==============================] - 138s 9ms/step - loss: 0.6180 - accuracy: 0.7448\n",
      "Epoch 102/105\n",
      "15918/15918 [==============================] - 50s 3ms/step - loss: 0.6157 - accuracy: 0.7457\n",
      "Epoch 103/105\n",
      "15918/15918 [==============================] - 53s 3ms/step - loss: 0.6169 - accuracy: 0.7452\n",
      "Epoch 104/105\n",
      "15918/15918 [==============================] - 57s 4ms/step - loss: 0.6138 - accuracy: 0.7463\n",
      "Epoch 105/105\n",
      "15918/15918 [==============================] - 50s 3ms/step - loss: 0.6131 - accuracy: 0.7468\n",
      "Epoch = 105,Validation Results = [0.7749558086253929, 0.663525]\n",
      "Epoch 106/110\n",
      "15918/15918 [==============================] - 139s 9ms/step - loss: 0.6113 - accuracy: 0.7479\n",
      "Epoch 107/110\n",
      "15918/15918 [==============================] - 52s 3ms/step - loss: 0.6121 - accuracy: 0.7471\n",
      "Epoch 108/110\n",
      "15918/15918 [==============================] - 51s 3ms/step - loss: 0.6108 - accuracy: 0.7479\n",
      "Epoch 109/110\n",
      "15918/15918 [==============================] - 60s 4ms/step - loss: 0.6088 - accuracy: 0.7489\n",
      "Epoch 110/110\n",
      "15918/15918 [==============================] - 52s 3ms/step - loss: 0.6093 - accuracy: 0.7487\n",
      "Epoch = 110,Validation Results = [0.7776408081694599, 0.6644125]\n",
      "Epoch 111/115\n",
      "15918/15918 [==============================] - 139s 9ms/step - loss: 0.6074 - accuracy: 0.7494\n",
      "Epoch 112/115\n",
      "15918/15918 [==============================] - 49s 3ms/step - loss: 0.6054 - accuracy: 0.7507\n",
      "Epoch 113/115\n",
      "15918/15918 [==============================] - 52s 3ms/step - loss: 0.6062 - accuracy: 0.7498\n",
      "Epoch 114/115\n",
      "15918/15918 [==============================] - 58s 4ms/step - loss: 0.6043 - accuracy: 0.7509\n",
      "Epoch 115/115\n",
      "15918/15918 [==============================] - 51s 3ms/step - loss: 0.6034 - accuracy: 0.7512\n",
      "Epoch = 115,Validation Results = [0.7770438033732212, 0.664224]\n",
      "Epoch 116/120\n",
      "15918/15918 [==============================] - 137s 9ms/step - loss: 0.6022 - accuracy: 0.7517\n",
      "Epoch 117/120\n",
      "15918/15918 [==============================] - 51s 3ms/step - loss: 0.6026 - accuracy: 0.7514\n",
      "Epoch 118/120\n",
      "15918/15918 [==============================] - 51s 3ms/step - loss: 0.6003 - accuracy: 0.7529\n",
      "Epoch 119/120\n",
      "15918/15918 [==============================] - 58s 4ms/step - loss: 0.5998 - accuracy: 0.7531\n",
      "Epoch 120/120\n",
      "15918/15918 [==============================] - 52s 3ms/step - loss: 0.5985 - accuracy: 0.7534\n",
      "Epoch = 120,Validation Results = [0.7819217445641613, 0.6650251]\n",
      "Epoch 121/125\n",
      "15918/15918 [==============================] - 137s 9ms/step - loss: 0.5978 - accuracy: 0.7539\n",
      "Epoch 122/125\n",
      "15918/15918 [==============================] - 51s 3ms/step - loss: 0.5972 - accuracy: 0.7540\n",
      "Epoch 123/125\n",
      "15918/15918 [==============================] - 52s 3ms/step - loss: 0.5967 - accuracy: 0.7543\n",
      "Epoch 124/125\n",
      "15918/15918 [==============================] - 58s 4ms/step - loss: 0.5958 - accuracy: 0.7547\n",
      "Epoch 125/125\n",
      "15918/15918 [==============================] - 52s 3ms/step - loss: 0.5949 - accuracy: 0.7555\n",
      "Epoch = 125,Validation Results = [0.777303135060558, 0.6659951]\n",
      "Epoch 126/130\n",
      "15918/15918 [==============================] - 140s 9ms/step - loss: 0.5940 - accuracy: 0.7557\n",
      "Epoch 127/130\n",
      "15918/15918 [==============================] - 51s 3ms/step - loss: 0.5933 - accuracy: 0.7559\n",
      "Epoch 128/130\n",
      "15918/15918 [==============================] - 51s 3ms/step - loss: 0.5917 - accuracy: 0.7566\n",
      "Epoch 129/130\n",
      "15918/15918 [==============================] - 59s 4ms/step - loss: 0.5920 - accuracy: 0.7567\n",
      "Epoch 130/130\n",
      "15918/15918 [==============================] - 52s 3ms/step - loss: 0.5914 - accuracy: 0.7567\n",
      "Epoch = 130,Validation Results = [0.8305404214652216, 0.6504634]\n",
      "Epoch 131/135\n",
      "15918/15918 [==============================] - 139s 9ms/step - loss: 0.5895 - accuracy: 0.7577\n",
      "Epoch 132/135\n",
      "15918/15918 [==============================] - 49s 3ms/step - loss: 0.5902 - accuracy: 0.7574\n",
      "Epoch 133/135\n",
      "15918/15918 [==============================] - 51s 3ms/step - loss: 0.5892 - accuracy: 0.7576\n",
      "Epoch 134/135\n",
      "15918/15918 [==============================] - 59s 4ms/step - loss: 0.5869 - accuracy: 0.7586\n",
      "Epoch 135/135\n",
      "15918/15918 [==============================] - 49s 3ms/step - loss: 0.5870 - accuracy: 0.7587\n",
      "Epoch = 135,Validation Results = [0.7739522952649027, 0.6680215]\n",
      "Epoch 136/140\n",
      "15918/15918 [==============================] - 138s 9ms/step - loss: 0.5863 - accuracy: 0.7591\n",
      "Epoch 137/140\n",
      "15918/15918 [==============================] - 50s 3ms/step - loss: 0.5848 - accuracy: 0.7596\n",
      "Epoch 138/140\n",
      "15918/15918 [==============================] - 52s 3ms/step - loss: 0.5859 - accuracy: 0.7593\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 139/140\n",
      "15918/15918 [==============================] - 59s 4ms/step - loss: 0.5839 - accuracy: 0.7599\n",
      "Epoch 140/140\n",
      "15918/15918 [==============================] - 50s 3ms/step - loss: 0.5837 - accuracy: 0.7603\n",
      "Epoch = 140,Validation Results = [0.7885088693981676, 0.664923]\n",
      "Epoch 141/145\n",
      "15918/15918 [==============================] - 138s 9ms/step - loss: 0.5828 - accuracy: 0.7606\n",
      "Epoch 142/145\n",
      "15918/15918 [==============================] - 52s 3ms/step - loss: 0.5821 - accuracy: 0.7608\n",
      "Epoch 143/145\n",
      "15918/15918 [==============================] - 50s 3ms/step - loss: 0.5811 - accuracy: 0.7613\n",
      "Epoch 144/145\n",
      "15918/15918 [==============================] - 59s 4ms/step - loss: 0.5812 - accuracy: 0.7614\n",
      "Epoch 145/145\n",
      "15918/15918 [==============================] - 52s 3ms/step - loss: 0.5791 - accuracy: 0.7621\n",
      "Epoch = 145,Validation Results = [0.7770631442907431, 0.67090404]\n",
      "Epoch 146/150\n",
      "15918/15918 [==============================] - 137s 9ms/step - loss: 0.5790 - accuracy: 0.7621\n",
      "Epoch 147/150\n",
      "15918/15918 [==============================] - 51s 3ms/step - loss: 0.5795 - accuracy: 0.7617\n",
      "Epoch 148/150\n",
      "15918/15918 [==============================] - 50s 3ms/step - loss: 0.5779 - accuracy: 0.7629\n",
      "Epoch 149/150\n",
      "15918/15918 [==============================] - 59s 4ms/step - loss: 0.5772 - accuracy: 0.7629\n",
      "Epoch 150/150\n",
      "15918/15918 [==============================] - 50s 3ms/step - loss: 0.5764 - accuracy: 0.7634\n",
      "Epoch = 150,Validation Results = [0.773415037257608, 0.67012644]\n"
     ]
    }
   ],
   "source": [
    "strategy = ipu.ipu_strategy.IPUStrategy()\n",
    "all_results = [[1000, 0]]\n",
    "split_train_val = int(np.floor(len(train_encoder_input) * 0.8))\n",
    "\n",
    "with strategy.scope():\n",
    "    # Create an instance of the model\n",
    "    model = get_model_seq(n_hidden)\n",
    "\n",
    "    # Get the dataset\n",
    "    train_ds = create_dataset(train_encoder_input[:split_train_val], train_decoder_input[:split_train_val], \n",
    "                              train_decoder_target[:split_train_val], batch_size, method='train', shuffle=SHUFFLE)\n",
    "    val_ds = create_dataset(train_encoder_input[split_train_val:], train_decoder_input[split_train_val:], \n",
    "                            train_decoder_target[split_train_val:], batch_size, method='val')\n",
    "    test_ds = create_dataset(test_encoder_input, test_decoder_input, \n",
    "                             test_decoder_target, batch_size, method='prediction')\n",
    "\n",
    "    # Train the model\n",
    "    adam = keras.optimizers.Adam(lr=0.00001, beta_1=0.9, beta_2=0.999)\n",
    "    model.compile(loss='categorical_crossentropy', metrics=['accuracy'], optimizer=adam)\n",
    "    epoch_ = 0\n",
    "    epochs_per_fit = 5\n",
    "    \n",
    "    while epoch_ < epochs:\n",
    "        \n",
    "        model.fit(train_ds, steps_per_epoch=len(train_encoder_input) // batch_size,\n",
    "                  initial_epoch=epoch_, epochs=epoch_ + epochs_per_fit)\n",
    "        epoch_ = epoch_ + epochs_per_fit\n",
    "        result = model.evaluate(val_ds)\n",
    "        all_results.append(result)\n",
    "        print(f'Epoch = {epoch_},' f'Validation Results = {result}')\n",
    "\n",
    "        if all_results[-1][0] < all_results[-2][0]:\n",
    "            model.save_weights(saved_model_path)\n",
    "\n",
    "    model.load_weights(saved_model_path)\n",
    "    pred = model.predict(test_ds)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Prediction horizon = 0\n",
      "accuracy_score = 0.8211357642472193\n",
      "classification_report =               precision    recall  f1-score   support\n",
      "\n",
      "           0     0.7356    0.5023    0.5970     21147\n",
      "           1     0.8382    0.9574    0.8939     98622\n",
      "           2     0.7658    0.4822    0.5918     19767\n",
      "\n",
      "    accuracy                         0.8211    139536\n",
      "   macro avg     0.7799    0.6473    0.6942    139536\n",
      "weighted avg     0.8124    0.8211    0.8061    139536\n",
      "\n",
      "-------------------------------\n",
      "Prediction horizon = 1\n",
      "accuracy_score = 0.7355377823644077\n",
      "classification_report =               precision    recall  f1-score   support\n",
      "\n",
      "           0     0.6543    0.4530    0.5354     27448\n",
      "           1     0.7649    0.9120    0.8320     86603\n",
      "           2     0.6494    0.4403    0.5248     25485\n",
      "\n",
      "    accuracy                         0.7355    139536\n",
      "   macro avg     0.6895    0.6018    0.6307    139536\n",
      "weighted avg     0.7221    0.7355    0.7175    139536\n",
      "\n",
      "-------------------------------\n",
      "Prediction horizon = 2\n",
      "accuracy_score = 0.7456283683063869\n",
      "classification_report =               precision    recall  f1-score   support\n",
      "\n",
      "           0     0.6842    0.5829    0.6295     31915\n",
      "           1     0.7813    0.8848    0.8299     78305\n",
      "           2     0.6826    0.5509    0.6097     29316\n",
      "\n",
      "    accuracy                         0.7456    139536\n",
      "   macro avg     0.7160    0.6729    0.6897    139536\n",
      "weighted avg     0.7383    0.7456    0.7378    139536\n",
      "\n",
      "-------------------------------\n",
      "Prediction horizon = 3\n",
      "accuracy_score = 0.7547657952069716\n",
      "classification_report =               precision    recall  f1-score   support\n",
      "\n",
      "           0     0.6994    0.7074    0.7034     38439\n",
      "           1     0.8080    0.8369    0.8222     65998\n",
      "           2     0.7087    0.6522    0.6793     35099\n",
      "\n",
      "    accuracy                         0.7548    139536\n",
      "   macro avg     0.7387    0.7322    0.7350    139536\n",
      "weighted avg     0.7531    0.7548    0.7535    139536\n",
      "\n",
      "-------------------------------\n",
      "Prediction horizon = 4\n",
      "accuracy_score = 0.7595602568512785\n",
      "classification_report =               precision    recall  f1-score   support\n",
      "\n",
      "           0     0.7079    0.8018    0.7519     47952\n",
      "           1     0.8292    0.7722    0.7997     48048\n",
      "           2     0.7518    0.6991    0.7245     43536\n",
      "\n",
      "    accuracy                         0.7596    139536\n",
      "   macro avg     0.7630    0.7577    0.7587    139536\n",
      "weighted avg     0.7634    0.7596    0.7598    139536\n",
      "\n",
      "-------------------------------\n"
     ]
    }
   ],
   "source": [
    "evaluation_metrics(test_decoder_target, pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
